{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7421cef5",
   "metadata": {},
   "source": [
    "### 初始化环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "18cdf54d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在jupyter notebook里env.render看不到窗口\n",
    "# 写一个helper类，用matplotlib刷新显示图像\n",
    "# 初始化传入env，调用helper的render即可\n",
    "from IPython import display # 导入display模块，用于在Jupyter Notebook中显示图像\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt # 导入matplotlib库，用于绘制图像\n",
    "%matplotlib inline\n",
    "\n",
    "class GymHelper:\n",
    "    def __init__(self, env, figsize = (3, 3)):\n",
    "        self.env = env # 初始化Gym环境\n",
    "        self.figsize = figsize # 初始化绘图窗口大小\n",
    "        \n",
    "        plt.figure(figsize = figsize) # 创建绘图窗口\n",
    "        plt.title(self.env.spec.id) # 标题设为环境名\n",
    "        self.img = plt.imshow(env.render()) # 在绘图窗口中显示初始图像\n",
    "    \n",
    "    def render(self, title = None):\n",
    "        image_data = self.env.render() # 获取当前环境图像渲染数据\n",
    "        \n",
    "        self.img.set_data(image_data) # 更新绘图窗口中的图像数据\n",
    "        display.display(plt.gcf()) # 刷新显示\n",
    "        display.clear_output(wait = True) # 有新图片时再清除绘图窗口原有图像\n",
    "        if title: # 如果有标题，就显示标题\n",
    "            plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5881877f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM8AAACeCAYAAACVU14NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAANZUlEQVR4nO3df2xd5X3H8ffHP64TO+QHcXBpQgKUtJBWGu0sCip/MLpqadYNtrGqLGpZF4lNohKdaDe2SVs7samVptKxoXVZ6UgnVEqhLYhBp/JDmyYVSEILLTDApAQC+eXEiW3s+Od3f9wn7o0bY/uJb47v9eclXfmc55x7/X1sf3TPPef4eRQRmNnsNRRdgFmtcnjMMjk8ZpkcHrNMDo9ZJofHLJPDM09I6pd0/lztK+kKSXvmpjqQFJIumKvXqwdNRRewEEl6FegAxiqa3x0Rb87k+RGxpBp12ew4PMX5rYh4pOgiLJ8P2+aJysMiSXdKul3Sf0rqk/SkpHdNse8mSc+n/d6Q9LlJr3uTpAOS9kr6dEV7i6R/kPSapP2SviZpccX2z6fnvCnpj6r/E6g9Ds/89Qngi8AKoAv4uyn2uwP444g4A3gf8FjFtncAy4DVwBbgdkkr0rYvAe8GLgYuSPv8NYCkjcDngI8A64Ffn6tO1ROHpzjfl3QkPb5/ku3fi4inImIUuIvyH/nJjAAbJC2NiJ6IeHrStr+NiJGIeAjoB94jScD1wJ9GxOGI6AP+nnJgAT4O/HtE/Cwi3gK+cIp9rUsOT3Gujojl6XH1Sbbvq1geAKY6SfB7wCZgt6T/lnRZxbZDKXyTX2cV0ArsPB5g4AepHeCdwOsVz9s9wz4tKD5hUOMiYjtwlaRm4DPAPcA50zytGxgE3hsRb5xk+95Jr7F2LmqtN37nqWGSSpI2S1oWESNALzA+3fMiYhz4N+BWSWel11ot6TfSLvcAfyhpg6RW4G+q1IWa5vDUvk8Cr0rqBf4E2DzD5/055RMRT6TnPgK8ByAiHga+SvnkQxcnnoSwRP5nOLM8fucxy1SV8EjaKOlFSV2Sbq7G9zAr2pwftklqBF6ifIFtD7AduDYinp/Tb2RWsGq881wCdEXErogYBu4GrqrC9zErVDWu86zmxAtse4APTt5J0vWUr3LT1tb2qxdeeGEVSjE7Na+++ird3d062bbCLpJGxFZgK0BnZ2fs2LGjqFLMptTZ2Tnltmoctr3BiVen16Q2s7pSjfBsB9ZLOk9SifLNhg9U4fuYFWrOD9siYlTSZ4D/AhqBb0TEc3P9fcyKVpXPPOn294eq8dpm84XvMDDL5PCYZXJ4zDI5PGaZHB6zTA6PWSaHxyyTw2OWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8NjlsnhMcvk8JhlcnjMMk0bHknfSBPC/qyi7UxJP5T0cvq6IrVL0m1pjOpnJX2gmsWbFWkm7zx3Ahsntd0MPBoR64FH0zrARylPALue8mig/zI3ZZrNP9OGJyL+Bzg8qfkqYFta3gZcXdH+zSh7Algu6ew5qtVsXsn9zNMREXvT8j6gIy2fbJzq1Sd7AUnXS9ohacfBgwczyzArzimfMIjyHCWznqckIrZGRGdEdK5atWr6J5jNM7nh2X/8cCx9PZDaPU61LRi54XkAuC4tXwfcX9H+qXTW7VLgaMXhnVldmXa4XUnfAq4A2iXtoTyt+JeAeyRtAXYDH0+7PwRsojyD8gDw6SrUbDYvTBueiLh2ik0fPsm+AdxwqkWZ1QLfYWCWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8NjlsnhMcvk8JhlcnjMMjk8ZpkcHrNM0/5Lgs0/EQExztjIMYb7e2hatIRS2/Kiy1pwHJ4atWf7/fS8soORwT463vdrrL7kd5BUdFkLisNTo9TQyFBfNwD9B35OjI+hRv86Tyd/5qlRS846D1T+9R07up+x4cGCK1p4HJ4aJIlFyztobG4BYHSwj6GjB6Z5ls21mYxVfY6kxyU9L+k5STemdo9XXaBS2wpazmgHIMbH6D+wq3wiwU6bmbzzjAI3RcQG4FLgBkkb8HjVhVJjM63tvxgib6D79bfZ26phJmNV742Ip9NyH/AC5SF0PV51gSTRdtZ5E+sDh15nfHSowIoWnll95pF0LvB+4ElOcbxqj1V96tra16HGZgCG+g4xMtBbcEULy4zDI2kJcB/w2Yg44beUM161x6o+daUzVtK0qA2A8ZEhBg97ZOPTaUbhkdRMOTh3RcR3U7PHqy5YU0sbi1e8M60Fffu6fNLgNJrJ2TYBdwAvRMRXKjZ5vOqiTfrcM3h4DzE+VmBBC8tMLkl/CPgk8FNJP0ltf4nHqy6cJNpWrQMJIhjs2cfY0AANrUuLLm1BmMlY1f8LTHXTlMerLtjiM1fTWFrM2NAAo4O9DPV10+zwnBa+w6DGNS9eSql1OVC+WPrWwd3+3HOaODw1rqGpROuqdRPr/fteLrCahcXhqXGSWNJx/sT64JH9jI8OF1jRwuHw1IHWledM/DvCcN8hRgf7Cq5oYXB46kDL0lU0Ly6fJBgbHmSw582CK1oYHJ460NTSSsvS43dp+GLp6eLw1AM10Lbq3InVwUOvQ4wXV88C4fDUgV/cYV2+HDfYs5fRoYFii1oAHJ460XrmahpLiwAYGexluP9QwRXVP4enTjS3LqO5dRkAMTbKWwdf8+eeKnN46kRDcwut7Wsn1n2xtPocnjpywsXSnr2Mj40UWE39c3jqhCRa29eihkYAhnq7GXnrSLFF1TmHp44sWnrWCRdLjx3ZX3BF9c3hqSONLa2UzliZ1sLDUVWZw1NH1NDIko53Tay/tf8VXyytIoenjky+w/pYb7eH4a0ijwxew/r6+ti6dSvHjh2baGsc6efy9nFamhsY6j/Mnf/6T+zvP/HdRxKbN29m3bp1k1/SZsHhqWG9vb3ccsstHDlyZKJtcamJrZ//XUorL2c4FnHfvbfzgx89c8LzGhoauOyyyxyeUzST0XMWSXpK0jNprOovpvbzJD2ZxqT+tqRSam9J611p+7lV7oNVGBwe5YnXVvJ/Ax9k1+DFtL5jY9El1a2ZfOYZAq6MiF8BLgY2piGlvgzcGhEXAD3AlrT/FqAntd+a9rPT6Lldr9HAGETw3rVttDQ3Fl1SXZrJ6DkB9KfV5vQI4ErgD1L7NuALlAd1vyotA9wL/LMkxducMx0YGGDnzp0Z5S9sBw8eZGzsl8dp273rR+x+5jZefGOA7je3Mzx64j4RwUsvvcTSpR5lZzoDA1PfnT6jzzySGoGdwAXA7cArwJGIGE27VI5HPTFWdUSMSjoKrAS6J73m9ZRnUWDNmjWsXbsWm51SqURDwy8fPPx872G+/p27p3yeJDo6Ovwzn4FSqTTlthmFJyLGgIslLQe+B1x4qkVFxFZgK0BnZ2d4vOrZGx4ezp6HdNmyZfhnPr2mpqkjMqvrPBFxBHgcuIzy1CHHX7lyPOqJsarT9mWA/7nE6s5MzratSu84SFoMfITyHD2PA9ek3SaPVX18DOtrgMfe7vOO5ZNEU1NT1sMzZ5+6mRy2nQ1sS597GoB7IuJBSc8Dd0u6Bfgx5cHgSV//Q1IXcBj4RBXqNqC9vZ2HH36Y0dHR6XeuIImLLrqoSlUtHDM52/Ys5QmtJrfvAi45Sfsx4PfnpDp7W6VSic7OzqLLWLB8b5tZJofHLJPDY5bJ4THL5PCYZXJ4zDI5PGaZHB6zTA6PWSaHxyyTw2OWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8NjlsnhMcvk8JhlcnjMMmk+jEcoqQ94seg6qqidSWN115l67t+6iDjpuMTzZXKrFyOibgcgk7TD/as/Pmwzy+TwmGWaL+HZWnQBVeb+1aF5ccLArBbNl3ces5rj8JhlKjw8kjZKejFNPX9z0fXMlqRzJD0u6XlJz0m6MbWfKemHkl5OX1ekdkm6LfX3WUkfKLYHMyOpUdKPJT2Y1s+T9GTqx7cllVJ7S1rvStvPLbTwKio0PGnCrNuBjwIbgGslbSiypgyjwE0RsQG4FLgh9eFm4NGIWA88mtah3Nf16XE95RnEa8GNlGcEPO7LwK0RcQHQA2xJ7VuAntR+a9qvLhX9znMJ0BURuyJiGLib8lT0NSMi9kbE02m5j/If2GrK/diWdtsGXJ2WrwK+GWVPUJ7b9ezTW/XsSFoD/Cbw9bQu4Erg3rTL5P4d7/e9wIdVp3M4Fh2eiWnnk8op6WtOOkR5P/Ak0BERe9OmfUBHWq7FPn8V+DNgPK2vBI5ExPH5HCv7MNG/tP1o2r/uFB2euiFpCXAf8NmI6K3cliY0rslrApI+BhyIiJ1F1zLfFH1v28S080nllPQ1Q1Iz5eDcFRHfTc37JZ0dEXvTYdmB1F5rff4Q8NuSNgGLgKXAP1I+3GxK7y6VfTjevz2SmoBlwKHTX3b1Ff3Osx1Yn87clCjPnP1AwTXNSjqevwN4ISK+UrHpAeC6tHwdcH9F+6fSWbdLgaMVh3fzTkT8RUSsiYhzKf9+HouIzcDjwDVpt8n9O97va9L+NfmuO62IKPQBbAJeAl4B/qroejLqv5zyIdmzwE/SYxPl4/xHgZeBR4Az0/6ifIbxFeCnQGfRfZhFX68AHkzL5wNPAV3Ad4CW1L4orXel7ecXXXe1Hr49xyxT0YdtZjXL4THL5PCYZXJ4zDI5PGaZHB6zTA6PWab/B58kPDGCWmjCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 导入gym库\n",
    "import gym\n",
    "\n",
    "# 创建CartPole环境，指定渲染模式为rgb_array，如果是在IDE中可以改为'human'\n",
    "env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "# 重置环境\n",
    "env.reset()\n",
    "# 创建GymHelper\n",
    "gym_helper = GymHelper(env)\n",
    "\n",
    "# 循环N次\n",
    "for i in range(100):\n",
    "    gym_helper.render(title = str(i)) # 渲染环境\n",
    "    action = env.action_space.sample() # 从动作空间中随机选取一个动作\n",
    "    observation, reward, terminated, truncated, info = env.step(action) # 执行动作\n",
    "    if terminated or truncated: # 如果游戏结束，则结束循环\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a86c4be",
   "metadata": {},
   "source": [
    "环境介绍 https://www.gymlibrary.dev/environments/classic_control/cart_pole/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "294a8fd4",
   "metadata": {},
   "source": [
    "### 策略梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce08e15f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入必要的库\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np\n",
    "import sys\n",
    "import time\n",
    "import random\n",
    "import collections\n",
    "from tqdm import * # 用于显示进度条\n",
    "\n",
    "# 策略模型，给定状态生成各个动作的概率\n",
    "class PolicyModel(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(PolicyModel, self).__init__()\n",
    "        \n",
    "        # 使用全连接层构建一个简单的神经网络，ReLU作为激活函数\n",
    "        # 最后加一个Softmax层，使得输出可以看作是概率分布\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(input_dim, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, output_dim),\n",
    "        )\n",
    "\n",
    "    # 定义前向传播，输出动作概率\n",
    "    def forward(self, x):\n",
    "        action_prob = torch.sigmoid(self.fc(x)) # 动作空间只有0和1，因此采用sigmoid函数直接输出动作概率\n",
    "        return action_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23fb1098",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义Policy Gradient Methods类\n",
    "class PGM:\n",
    "    # 构造函数，参数包含环境，学习率，折扣因子，经验回放缓冲区大小，目标网络更新频率\n",
    "    def __init__(self, env, learning_rate=0.001, gamma=0.99):\n",
    "        self.env = env\n",
    "        self.learning_rate = learning_rate\n",
    "        self.gamma = gamma\n",
    "\n",
    "        # 判断可用的设备是 CPU 还是 GPU\n",
    "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        # 定义策略网络\n",
    "        self.policy_model = PolicyModel(env.observation_space.shape[0], 1).to(self.device)\n",
    "        # 定义Adam优化器\n",
    "        self.optimizer = torch.optim.Adam(self.policy_model.parameters(), lr=learning_rate)\n",
    "\n",
    "    # 使用策略模型生成动作概率分布并采样\n",
    "    def choose_action(self, state):\n",
    "        # 将状态转换为tensor输入模型\n",
    "        state = torch.FloatTensor(np.array([state])).to(self.device)\n",
    "        probs = self.policy_model(state.to(self.device))\n",
    "        \n",
    "        # 生成分布后采样返回动作\n",
    "        m = torch.distributions.Bernoulli(probs)  # 伯努利分布\n",
    "        action = int(m.sample().item())  # 转为标量\n",
    "        return action\n",
    "    \n",
    "    # 模型更新\n",
    "    def update(self, buffer):\n",
    "        states, actions, rewards, next_states, dones = zip(*buffer)\n",
    "        states, actions, rewards = list(states), list(actions), list(rewards)\n",
    "        # 对奖励进行修正，考虑未来，并加入衰减因子\n",
    "        running_add = 0\n",
    "        for i in reversed(range(len(rewards))):\n",
    "            if rewards[i] == 0:\n",
    "                running_add = 0\n",
    "            else:\n",
    "                running_add = running_add * self.gamma + rewards[i]\n",
    "                rewards[i] = running_add\n",
    "\n",
    "        reward_mean = np.mean(rewards)  # 求奖励均值\n",
    "        reward_std = np.std(rewards)  # 求奖励标准差\n",
    "        for i in range(len(rewards)):\n",
    "            # 标准化奖励\n",
    "            rewards[i] = (rewards[i] - reward_mean) / reward_std\n",
    "\n",
    "        # 梯度下降\n",
    "        self.optimizer.zero_grad()\n",
    "        loss = 0  # 初始化损失\n",
    "        for i in range(len(rewards)):\n",
    "            state = states[i]\n",
    "            action = torch.FloatTensor([actions[i]])\n",
    "            reward = rewards[i]\n",
    "            state = torch.FloatTensor(np.array([state])).to(self.device)\n",
    "            action_prob = self.policy_model(state)\n",
    "            c = torch.distributions.Bernoulli(action_prob)\n",
    "            # 加权(reward)损失函数，加负号(将最大化问题转化为最小化问题)\n",
    "            loss = -c.log_prob(action.to(self.device)) * reward\n",
    "            loss.backward()\n",
    "        self.optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20a680b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0: 11.0                         \n",
      "Episode 200: 353.0                                \n",
      "Episode 400: 500.0                                \n",
      "Episode 600: 500.0                                \n",
      "Episode 800: 500.0                                \n",
      "Episode 1000: 500.0                                \n",
      "Episode 1200: 500.0                                \n",
      "Episode 1400: 500.0                                \n",
      "Episode 1600: 500.0                                \n",
      "Episode 1800: 500.0                                \n",
      "100%|██████████| 2000/2000 [08:45<00:00,  3.81it/s]\n"
     ]
    }
   ],
   "source": [
    "# 定义超参数\n",
    "max_episodes = 2000 # 训练episode数\n",
    "max_steps = 500 # 每个回合的最大步数\n",
    "# batch_size = 10 # 采样数\n",
    "\n",
    "# 创建DQN对象\n",
    "agent = PGM(env)\n",
    "# 定义保存每个回合奖励的列表\n",
    "episode_rewards = []\n",
    "\n",
    "# 开始循环，tqdm用于显示进度条并评估任务时间开销\n",
    "for episode in tqdm(range(max_episodes), file=sys.stdout):\n",
    "    # 重置环境并获取初始状态\n",
    "    state, _ = env.reset()\n",
    "    # 当前回合的奖励\n",
    "    episode_reward = 0\n",
    "    # 记录每个episode的信息\n",
    "    buffer = []\n",
    "\n",
    "    # 循环进行每一步操作\n",
    "    for step in range(max_steps):\n",
    "        # 根据当前状态选择动作\n",
    "        action = agent.choose_action(state)\n",
    "        # 执行动作，获取新的信息\n",
    "        next_state, reward, terminated, truncated, info = env.step(action)\n",
    "        # 判断是否达到终止状态\n",
    "        done = terminated or truncated\n",
    "        \n",
    "        # 将这个五元组加到buffer中\n",
    "        buffer.append((state, action, reward, next_state, done))\n",
    "        \n",
    "        # 累计奖励\n",
    "        episode_reward += reward\n",
    "        \n",
    "        # 更新当前状态\n",
    "        state = next_state\n",
    "        if done:\n",
    "            break\n",
    "        \n",
    "    # 更新策略\n",
    "    agent.update(buffer)\n",
    "    # 记录当前回合奖励值\n",
    "    episode_rewards.append(episode_reward)\n",
    "\n",
    "    # 打印中间值\n",
    "    if episode % (max_episodes // 10) == 0:\n",
    "        tqdm.write(\"Episode \" + str(episode) + \": \" + str(episode_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "442cc706",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAAsKElEQVR4nO2deZwdZZX3v6c7nX3pLJ3O0gmdjSVEQkIbwmogsgYIoCCoEBiUV8VXHXSGDI6OoqNxfJFR3EDhFRX3ZUDAYQeHLRi2AAmYhUAIWTohi4QsdPczf9y63bfvrXtv7ds938+nP133qe3UU0/96tR5NjHGoCiKomSLurgNUBRFUYJHxV1RFCWDqLgriqJkEBV3RVGUDKLiriiKkkFU3BVFUTKIiruiRICIXCIij8Rth1I7qLgriqJkEBV3JdOISJ9aOKeiFKPirmQOEVknIleJyHJgt4gcKyKPicgOEXlOROZZ250gIs8X7HeviPy14Pf/iMjZ1vJiEVkjIn8XkRUick7BdpeIyKMicp2IbAO+JCIjReR2EdklIk8CU6K5ekXJoR6GklUuBBYAXcBy4CLgv4H5wO9F5GDgCWCaiIwCdgKHAR0iMgToANqA/7GOtwY4DtgEnAf8XESmGmM2WuuPBH4FNAMNwP8H9gJjgUnA3cArYV6wohSinruSVb5jjFkPfBi4yxhzlzGmyxhzL7AMON0Yswf4K3A8cATwHPAocAwwF1hljNkGYIz5rTHmDesYvwZWAXMKzveGMeZ6Y0wHsB94H/BFY8xuY8wLwC2RXLWiWKjnrmSV9db/A4DzROTMgnUNwIPW8sPAPOB1a3k78B5gn/UbABG5GLgSaLWSBgOjbM4H0ETu2SpMe9XzlSiKB1TclaySH+50PfAzY8xHy2z3MHAt8BqwhJy4/4icuH8PQEQOsNLmA48bYzpF5FlAbM4H0E4urDMBeMlKm+jzehTFFRqWUbLOz4EzReQUEakXkf4iMk9EWqz1jwEHkQuxPGmMeZGct38k8Bdrm0HkxLsdQEQuBWaUO6ExphP4A7mK1YEiMh1YFMK1KUpZVNyVTGPF3RcCV5MT5/XAP2GVfWPMbuBp4EVjzH5rt8eBV40xW6xtVpDz7h8HNgPvIhebr8QnyYVuNgE/IVfBqiiRITpZh6IoSvZQz11RFCWDqLgriqJkEBV3RVGUDKLiriiKkkES0c591KhRprW1NW4zFEVRUsVTTz211RjTZLcuEeLe2trKsmXL4jZDURQlVYhI2Z7PGpZRFEXJICruiqIoGUTFXVEUJYOouCuKomQQFXdFUZQM4kjcrWnLnheRZ0VkmZU2wpqWbJX1f7iVLiLyHRFZLSLLRWR2mBegKIqilOLGcz/BGHO4MabN+r0YuN8YMw243/oNcBowzfq7HPhBUMYqiqIozvDTzn0huRlsIDeF2EPAVVb6T01uuMknRKRRRMYWzDWZSZ5+bTvnfv8xZk9s5Ogpozh71ng+/OOl7O3o5Psfms3RU3om7XlrXwff+PNLzBg/lHNnt3DitQ8xadRgnn1tOw31dcydPJIpTYPY29HFE2u38ebu3Ei0cyePZPe+Dv78wiYuObqVIf37cP0Dq5k2ejCnzRjTfXwD3LdyCycdMtrW1of/1s6BzUPY19FF68iBrq5zb0cXy9a9ybFTR7Fi4y76N9Rzx/LcrZ04YiBnzRxHnZTut6+jiyfXvcm4xgG8sWMPU5oGM25Y/5LtVmz8O0MH9KGlcQB/WbWVIyeNoF+f6j7I6zv2sGtPB9PHDgHgiVfe5MlX3uTMmeP403NvcM6s8Tz92nbq64SNO/ay551ORgzqy8FjhjB19GAaBzTYHvfRNduYPGoQ67e/zZzWEbbbPLJ6K0ccMJwBDfVV7azEU69tZ9KoQYwY2LfsNr94cj2jh/RjxcZdzGkdwdzJlW3q31DP9Q+sZt5BTbyydTdnzRyHze2Jlde372HX3g6mjh7M42u28p4Dbfvk8Mz6HUwYMZBRg8rnj1Pa39rHH5/ZwAfaJjBsQANr2ndz5/MbOb+thRc27GL+IaMjy6f5hzQzc0Jj4Md1NOSviLxCboYaA9xgjLlRRHYYYxqt9QJsN8Y0isgdwBJjzCPWuvuBq4wxy4qOeTk5z56JEyce8eqr6Z6FrHXxnRXXr1uyoHv5ruc38olbnwbgXxccwlfvXFmyvQi4HY1ZrNJYuJ8UlVC7YxZvUwknNtkdz+l5vdqX389LvvmxpVJeu8HJccpdl5N7XGn7uIkynyuds5oNYfGVhTP48NwDPO0rIk8VRFN64dRzP9YYs0FERgP3ishLhSuNMUZEXD1SxpgbgRsB2traMj+o/J+ee4P5h4xmYN8+vNPZ1Z3et4xX+srXFzDvmw+ybtvbZY/ZPLQfm3ftA+CahYdy8VGtAHz9zyu54eG1XHXqwXx83pRe+zy6eisf+vHS7t/Lv3QyQ/vbe6125G36wyeO5tzvP1ay/n2zW7j2/Jkl6fOvfYg17btLrrGY/EvyN//nKM6/4XEOah7C3f94fFW78vu98vUFdHYZplx9l6PrqWTLCxt2csb1j3T/fuSqE2gZ3vtL55nXtnPO9x+jZfgAHrnqRFfnLGTrW/to++p9ZW0BeGTVVj5809LeaRVsmjhiIAMa6nl589+71727dTi//djRnu0Mg/y9mzWxkWde28GtHzmSY6aO6rXNzrffYeY19wDl88fLOQGWnPsuFv/h+V7rL3j3BJa87zDf54kTRzF3Y8wG6/8W4I/kpiTbLCJjAaz/W6zNN5CbOzJPi5VW0/zfXz7DF/7rRQCkwCVoGT6g7D7V3nh96pLX2MlUtVqxoy5p7nTCqK/X/HFLVXUQkUEiMiS/DJwMvADcTs+8kIuA26zl24GLrVYzc4GdWY+3O2Xjzj0laX4mwir0+nVCrfCREAXYrp4iCIpNlsRF3J1RH2LeZ/W96iQs0wz80SrYfYBfGGP+W0T+CvxGRC4DXgXOt7a/CzgdWA28DVwauNUZopIoVytzfcopgsn/S47ie7UkSddQ6X74fbl6fXFU2i9JeecGuyuqD+vth/0LLwvOUlVxN8asBUqCqMaYbcB8m3QDXBGIdTWAnzJU+FwfPGaIs32q/HZ8bo/7pYkSrzfEiw5Ru1JDJUEt68gEQUbzPnlB2xqgsCz5maC80OM4cvLIkhMk6RPcrSXi4xrCumo7W/Kes1/hdxJzd3qObpsSdP/9Uheq526TloGsU3GPELsC01VB26vJftOQfgBcfvxk70YpZSkWxzAfeK9hh2p7lYRtEixa3aZGbGOYdSlxouIeO94997wgHDVlZJUtCygJNSS7YKc1buyWsG6Dny/DqInLVLusT1G2lUXFPQYKH2Q/FarlmDFuGAAHjRns8QjVSfpLIQjcOL1+xcBrU8hKuxlMTdwnv2Q1ixIxzV4tE4aDcObMcRw6biiTm8IT97Dp+UL3EHMP6GEtOU6oFaoOYu4ejhvhJfjGTz1LEOetlpY21HOPkHyhLfTyuny4fJX2LCfsJXFkz2f3xh8+kazekW6wrVDN//ddoepv/0L8vBjjpNqjcNy0Ufz7OTMCP2/a8skp6rnHTKUCnYGwX6qJskLVczv3jAqTHT+77MhQjut0XKG0oZ57zGSgDDkiC9cZZsw9LKJsq++X7rBMgm1MEyruMVCoA5VaM4RRxoN62MN+/pLwgJfmVcxGeYgNx22yG2JrLWOTSWnKt3KouEeIXYHprNTQPYF4tTbKJsxRirB6m+knq7dOxT1C7DwTP+Ke1UKZFNLU0qSQSi+aJMfoe1rLxHPeQpIaZnODinvMRF2h6vfB8bp/JgYOS6AuJtCk1JHkF54ftLVMhPTMlNQjWH6aQgaB14KdRKELGjfD5QZxG6886UDmTi7f29jtvTKYVApX1HUbWW3nruLukeWv7+DQccN8D0WaspC775i72z3iFadox2X51Pxp7ndyWaGaBdEKmqxmiYZlPPDc+h2c9d1Huf6BVb6P1ZmF4J4LYm9x4gNbDy//EkrgZaXRa48Djbkr3WzatReAF9/Y5ftYXRVc92qPppfyVyyubkUparmIM+aeRMF2QqGop/ESos/3NOZSdVTcPZAvCkGMuBd3zN0r7uO/6SduCaj05eDnGEkhvnbuztLShoq7B4IMLdRKzN0tcQ0i1cuG4t8pe+INpkSlNFRTSlZzRMXdB0F4GpXCMqE0hYypJEfZiSks7GxPmd4nkscWn8h9Vx4fYzv30jOm9IO6F9paxgP5BjKe224X7OinQjUOXSk856RRg3hl6+4YrIgGv/UTUVAppCAp8dPHNQ4AkjVZRxZQz90D+YcniHh5pWMkudCJwIOfm+d4+zQ6QqU9VJN8RxSvaMxd6SbIhzxqb6U0juxu/8hi7hGdxw0VZz2KIGNsw0LV9klRO/e4xulJcp74QcXdC909TV3uZlOK0jZwmFfS+Pxk9aFPKvGFZbJ5o1XcPZAvCl7DMoVttyuJexJl3/fYNNl8jjJ7XTVBRu+dirsHgmwSF/Xs9G7GS0kCsXZiinAmJq+krXlmJXouJeKxZSI9W3SouHsgyMJQKSoTRg9VvyTxayIq4n4RehHyrApXkGTpBVmIirsHvLaW6enZ2pNWa2PLOCX/wMXaiSkFlZGVTDKYzApXkGQ1h1TcPZAXnEA6MdkcZNrowf4PXBZ/Rdn/ePDpfZSy0IlJxb6UrGaJirsHesZlz/3fuecdz8ey66Fqiv6XtcPzWQuO4fEgUT0QSZqsI22k+UUaJVnNJxV3D3SHVyzhuWP5G56PlbaWkNXMPX3GWE/7JZGkTZDttLNNYVo2ZStY1HNXevDYzj1Pr5i7jyF/k8h7pzcHcpyesWiSkwsV49sJfXuVto5KLj15GHELskjPFh2OxV1E6kXkGRG5w/o9SUSWishqEfm1iPS10vtZv1db61tDsj02gu2hGvc0e+Fu73e/OEnF2DKVpv7DJPalkygSeF+DwI3n/mlgZcHvbwDXGWOmAtuBy6z0y4DtVvp11naZojvmHsCxom8tE8z5anHuVbuwTJJnYoLk2mVHfO3cU5RJLnAk7iLSAiwAfmz9FuBE4HfWJrcAZ1vLC63fWOvnS9zByoDpadLosimkzUuhUsw9iU5XEm0Ki6QVWvdfWaXjQib5SYwtLJPgPPGDU8/9P4F/Brqs3yOBHcaYDuv368B4a3k8sB7AWr/T2j4z5N9V+cLo9M2/ccfekjS71jLhlrXihz2ZJTuhZiUPuwrVxL2Wkk1Wc6uquIvIGcAWY8xTQZ5YRC4XkWUisqy9vT3IQ4dOsQfutLneO11dJWl27dzzKeH0UPXnFWX1QbAjEy+YFF1DbGGZTNzoUpx47scAZ4nIOuBX5MIx3wYaRSQ/2UcLsMFa3gBMALDWDwO2FR/UGHOjMabNGNPW1NTk6yKixmtYxo60NYXMk9HnwTVpy4ckm6thmWCpKu7GmH8xxrQYY1qBC4AHjDEfAh4E3m9ttgi4zVq+3fqNtf4BE3eTkIDxW6FamB2VptmrakcAe2W0XAdCGsIb1YQp+VcQP1nNIz/t3K8CrhSR1eRi6jdZ6TcBI630K4HF/kxMIh6HH7DZ3k9rmY8eNxmAw8YP82dEBLg9axKENa0eXfrtjjosE+npIsPVHKrGmIeAh6zltcAcm232AucFYFtiKfbc3QrRf963qnvZznF3erRjp41i3ZIFrs6thENS5/9MWi/bZFKaJ1mINWgPVQ90FwWPJWDDjj3dy5UqVMMhmIfbrUakUVIKbb7uAzNjs8MrOi6PM7L6vlNx90Bdvimky/3Wbt1dkhZ3dUTS56tMgkA1DenHObNabNfFNe+nU0raucdkR5LJwmifdqi4e6B4VEg/AlQrc6gq4VApzJKEeos0kNVQlYq7D4LwKjtLm74rFHjEKlCuyahWhYZddmnMvUYJcrIOu7BMlM+mV69FRbc3UYiB3SncV6gGZU12yGqeqLh7oDgs40foKk3VlwHnoQR9KURHEuor0oBdmcyC4Ku4+yCYUSEDOIgSKpW88ihHhfRyiiyIVNhkNY9U3H0QyPADFabZy1KZ03dYOFQSJvuvpOSWqriGH7BDY+6KbzbtKh0pMlwCGs89uRoROEm5Vqd3TkNf7kjK/Q0aVz1UlRzdTSA96GSxt796y1sl22SxrBVe04VzJtI6cmBstmSJakKeJqFP0mQdWRB8FXcfeKmwcvNCCOfLMN5SKwJfP/ddsdoQJFGKQBAx9ySLlo4KGSwalvGBF8+9UuuYaEjJwGEZfeCU5GFX1mJ/TANAxd0DeY/dy/130yE1yfqWZNtqiSy9BJMUlskCKu4+8NJaJn7PPZ6CnM3Hp4fYb2sBlVvQKMXY5VcWXpoq7j7w8jzHLwLhGfCji9uqbpPGgcPSiuadMzKg47aouPugu4eqi9LhxnNP06PZr08dJ01vjtuMyMnf+qR6elkdFCtINOaudOPnxqc9LNPdwSpkzeju+ZlZvyp8strEL3iymSkq7j7Ix9zd6HVWKlSVZGAbL47ejFSjMXelBG8x97g9d39EXeY1buyPDGhU6GQ1j1TcPZCXG2/t3J0fP4tkLcySZA/PYEo7MWUs/5XyaA9VH+S9yiArVBcddQAXH93qw6pw6bHenUh47cSkYlQZzR//ZLXSWcXdB2H0UP3ywhkerVGU3ohKvyOymkcalvGBlwHEUh5yL/sgVHN+0vgA1dflrB45qG/VbeO6r24rAzPqpPoiq3minrsHiitF3TRvrDQh9g0XHeHZJiV4Rg7ux3+87zDec1BT3KYoIZLV7xsVdx/0NIV0Lu6VXgSnHDrGt01h47edu9P9kuJNnf/uCVW2iG4mJi8Ux5OTaifoqJBBo2EZH+Sd8EreeDFpD8so8eFlgmyldlFx90DxQ9bhQtzj76HqDxWTdJGm+xXbqJBpyiQXqLj7IN8UMqiYezSEc/6Uv7NSge2MqLbKVF6tshpf9kNWm0KquPugJyzjfp+04nfy7qw9RknWBbtOTEopWc0iFXcP3P3iJqDHW+3scq7u8Q8/EE9Rjvuqs4DmYfB86sSpmX0Bqrh74IaH1/b6nS7P3Z8B5Z6D2C+rRqmkS7YhmIwKmVeuPPmgzIaqVNx9kZO0Thfe+M4974RlTKJx+/h0dxBLyWsjig+ybEpQ/NSs5y4i/UXkSRF5TkReFJEvW+mTRGSpiKwWkV+LSF8rvZ/1e7W1vjXka4iNvBfe5cIdP/+Gx0OyxilBjeee0SciA+itcUdWs8uJ574PONEYMxM4HDhVROYC3wCuM8ZMBbYDl1nbXwZst9Kvs7bLNGlv3hglbjsxJf2TOe6ZmKqdt7hoJjs3YyKjmVJV3E2Ot6yfDdafAU4Efmel3wKcbS0vtH5jrZ8vGXXz8qJeS9Je9kbWUiYomSLpDoRXHMXcRaReRJ4FtgD3AmuAHcaYDmuT14Hx1vJ4YD2AtX4nMNLmmJeLyDIRWdbe3u7rIuIi/jbrSq1TzW/KplsVLFnNI0fibozpNMYcDrQAc4CD/Z7YGHOjMabNGNPW1JTOgZncxNoVJQlk9CPaF1nNEVetZYwxO4AHgaOARhHJDzzWAmywljcAEwCs9cOAbUEYmzTy2h5WyD1Nrw7nrVqy9SglUSwLLdLqoOok8R4GgZPWMk0i0mgtDwBOAlaSE/n3W5stAm6zlm+3fmOtf8DE33MnFNw0gVTyaJ4p9sQ2KmSkZ4sOJ0P+jgVuEZF6ci+D3xhj7hCRFcCvROSrwDPATdb2NwE/E5HVwJvABSHYnQjCDstktdAp0VE6h6pSTEYd9+riboxZDsyySV9LLv5enL4XOC8Q6xJOT2sZ9UYVxS+xjQppc74sfJRrD1UfdDvuGSgIXhncL+cfZDVumUYK70X/hnoAxg3rH5c5ySejRVfF3SfX/GkFL76xK24zYuPnHzkSgEF96ytul9X3X5J1ITcqZM7Cg8cOBbIbgvCD23lo04JOs+eTmx99JW4TYmX0kH5xm6AovsiAjtuinrsSCNU886wPHJZECmPJGW2wFgh2IcUsZJeKu+KLLHy+BkFaxEBvVylZzRMV95ojbhVy9iilZeCwJKI55o6sxtxV3B2ybN2brNxYuxWn5ciLr3PPNe6XS7B0v4QSLgY6VHN5supAaIWqQ97/w9w47OuWLIjZEr8EW5BVK5S0Y1eG0xJmq4R67opSA2RBrBR3qLgnmCtOmBq3CVVRxz06gtBnvV+laMxdiZzz2iakJgxUraldVh3HJMZrsyBMUZLEexgEKu4p5Vvnz4zbhBzZfC4SiZ+szurLNQiy+jJUcY+RBe8a63nfhvpk3DqnXk9Gn59Y+d3HjnK/k96IErKaJclQiBrFT+/LpHkbte4ZxlFh2dY6IvqTZpCsNg9VcU8pSWn94P65yOaDlHR0+IHyZLVEqrinlIb6tBbJbIlMkjsx2X0ZZrXy0A9JvHdBoOIeI36cqX59Kg+xGxVhPRc6cJgSFRqWURJF3z4Ju3WqwYlDvfTaJmEKUVv489yTcevC8np04LBwyKiTqtiQDIVQXDNsQEPcJgDOwzLq2EeHvhAVUHGPFa/x5K+cPYNpzUMCtsYfKt7Jwa5caWOZ2kPFPYX46fwUtAw7/cxXXzIeiptA6n2oHVTcY8SrN6UPaHLobtWjnrGSMFTcYyQePQh4PPfuyTqcXo2+mqIkX+GdhialZx8+HoAJwwfEbEk20Mk6FH+41urki4wbktiJya5CdcLwgcA2DhqTrLqaQi4+6gA+eOTExIyblHZU3FNIkoRESR52XvpRU0Zy4ZyJHNYyLAaLnCEiKe55nTxU3GMkjXHaYpP1RZMeZk5ojNsEJUL0+ydWUqjuZcjOlShKNlBxTyFxdlIpPrM67oqSTFTcFV9kddClLJCGFjJKeKi4x0gWYu7d6Sm8lrShWay4oaq4i8gEEXlQRFaIyIsi8mkrfYSI3Csiq6z/w610EZHviMhqEVkuIrPDvoi04vlhTZCznCBTlCJ0jJnaxonn3gF81hgzHZgLXCEi04HFwP3GmGnA/dZvgNOAadbf5cAPAre6xvjuB2dx1akHx20GEISYq+B4RXMuPL59weFxmxA4VcXdGLPRGPO0tfx3YCUwHlgI3GJtdgtwtrW8EPipyfEE0CgifgZDSRRzv3Z/5Oc847BxHDw2mZ1P3IfcNbigJI+FVu/YLOEq5i4ircAsYCnQbIzZaK3aBDRby+OB9QW7vW6lFR/rchFZJiLL2tvb3dodG5t27Q3sWF7ntUxiHaZW3ilKsnAs7iIyGPg98BljzK7CdSanUq6ebmPMjcaYNmNMW1NTk5tdlQShcV1FSSaOxF1EGsgJ+63GmD9YyZvz4Rbr/xYrfQMwoWD3FitNKSJuXzeJXwCKogSDk9YyAtwErDTGfKtg1e3AImt5EXBbQfrFVquZucDOgvCNUoCrqEzBtn40uU9dbu+jp4zkla8v8HGk3mhTSEVJFk7GljkGuAh4XkSetdKuBpYAvxGRy4BXgfOtdXcBpwOrgbeBS4M0WPHHYS3DuPKkA7ng3ROqb+yAtHv/x0wdyRfOmB63GYGioTIFHIi7MeYRyjuL8222N8AVPu1SQkJE+NT8aXGbkRg+fOQBHDxmaNxmBIpWbiugPVRjxesjmMQu/yonSpbIQphRh/yNETdNIdPujaXbeiVrfGXhoTz0cnqaYHtBxT1GsuAdKEoaueioVi46qrXs+gR+HLtGwzIxMmtio6f90lju0mhz2kn7157iDxX3GPn4vCnc+4/HO9q20MtP5CObSKMUxRtZ+KpWcY8RQZjWnMwxY5yS/3ytq9GSlBeBJIqBNomsbTTmrviiX596PjV/GqfNGBPocbtFUz8JfOF1/KJaJwsxdxV3xTdXnnRg3CbERl4EsiAGSrao0Y/p9FFrDli3aGpowRdJ7BOhRIOKewWeXb+Dzq7wVNVryEE/tRVFqYaKexmeeW07Z3/vUa5/YFXcpiiKorhGxb0Mm3bmJuV4aePfHW1/6TGtIVqjKIriDhX3MrgNfPStr2Pp1fN56HPzwjBH24woiuIKbS1ThDGGlQXeuuP6KIHmof3DMaoIFXpFUaqh4l7ErUtf41//6wUunDPR1X5uWnVcOGcinV1dDGiod2uekjCS3IlJqW1U3ItYsTE3Peyr23a72s9Ni7PZExs5ry2YyTIURVHs0Jh7QITdmnj3vo6Qz6B4QTsxKUlFxb2I4s9rpw9t2A93/4aeW6XhHMUJOnRDbaNhmTI8tmabq+3dxNy9PHKnHDqGmy9pY96Bo6mrUzdRUZTKqLgHRNieu4hw4sHN4Z5EyRQ6dENto+IeB0Wu+2XHTmJc44B4bFEUJZOouJfQW3mdej9+fKQvnDHdx96KoiilaIVqULiIy2hFl6IoYaPiXoTXziga3VQUJUmouFfDoWq7eSdob0ZFUcKmpsX9rX0dfPeBVb3GbPcqvDrGuqIoSaKmxX3Jn1fy/+75G39+YaPvY1Wb1OP4A5t8nyMJXLNwBpObBjGuMZpB0hRF8UZNt5bZva8TgP0dXRW327O/s+qxQpywKVEcf2ATD3x2XtxmKIpShZr23J0gwOyv3Ft1u3Jhmes+MJOpowf33jYIwxRFUSpQ0+LuNE6+5x0nnrv9sc6Z1cJ9V76n17kOn9Do6LyKoiheqWlxt8NpG/TpY4f2+u00LHPzJW0cUrSvoihK0FQVdxG5WUS2iMgLBWkjROReEVll/R9upYuIfEdEVovIchGZHabxUXDHcvvK1j71vdtI2nnuBzUPKUmr07FhFUWJACee+0+AU4vSFgP3G2OmAfdbvwFOA6ZZf5cDPwjGzHAQG6H13hSyNO3UGWMcnbMW0HoGRYmWquJujPkL8GZR8kLgFmv5FuDsgvSfmhxPAI0iMjYgW0Pjyt885/sY1ZpCKoqSHoYNaIjbBN94bQrZbIzJxys2AfmxaMcD6wu2e91K89+QPCKcSPSowX1L0spVqCo5avN7RUkjXz7rUD7w7vRPg+m7QtXkmoG4VjYRuVxElonIsvb2dr9mREq/PqUzIdllgMq9oqSPRUe30j8Ds515FffN+XCL9X+Llb4BKHzltVhpJRhjbjTGtBlj2pqaou29uWXXXrbs2sv+zsqdl8pRVwdXnDC1V5oOP6AkkYvmHgDAkZNGxmyJEjVexf12YJG1vAi4rSD9YqvVzFxgZ0H4JnaMMXR1GeZ87X7mfO1+z651nQinHDqGb50/szuty+49oYKvBMjMCY2cNXOcq33mTBrBuiULGDNMh4uoNZw0hfwl8DhwkIi8LiKXAUuAk0RkFfBe6zfAXcBaYDXwI+AToVjtkR8+vJbJV99lu25t+1vsfafTsx7bxdxV2pUgaaiv4zsXzorbDCUlVK1QNcZcWGbVfJttDXCFX6PC4talr/b6Xdhh6cRrH+a0GWMY2Ld6HXO+Zcwph44Bci1ttLGMoihJoqZ6qBY718WhlAde2uKoh2qXpeSD+vXh2vNmWse28dxtDqWtRhRFiYKaGhWyWICLhXxfldEh8wzq15NtddbrsdM2LKPuvFdGDuoHwImHjHa0/SNXndA9yqeiKLUm7kW/bUMpDvR4aEEHh/xwAhqWCZamIf346+ffy4hBpX0K7GgZPjBkixQlXdSWuJvKv51SGFrpEffSg118VKvvc9UyTUP6xW2CoqSW2oq5F7nlz2/YYbNNdQoH/8ov28Xcm4dq8zNFUeKhpsS9OHSyedc+bwcqcN3rrGXbdu6Fu2hNqqIoEVJT4h5UaKRQp8UmLDOwbz0nHNS7122th2UuOWYSAKMz9jWTv6+1fn+V5FFTMfeguhX1Dsvk/hd+Fay4pniE5B5q1YO/aO4B3V3hFUUJn5ry3Le+tb/qNpXGiDl31nigt0BXqlBVsk++LNTqS1tJLjUj7s+/vtP3MaaPy02P10vcrRxUcVcUJUnUjLi/+uZuR9s11JfPknx8vTAsI92tZXwYpwTO0qtLRsdQlJqiZsR9cD9n1Quvbnu77LrZExtpHtqPz7z3wO40p2EZ7a0aLVE1Q82Xq7mTdUhdJVlkvkJ159vvMGxgAwMcDr7/5LriGQV7GDaggaVXv7dXWr5CVT332mT4oL7c/9n30DJ8QNymKEovMu25P7ZmKzOvuYcHX9pCfZ3/Gi+7ya21QlWZ0jTYdnYuRYmTVIv7m7v307r4Th5ZtbVX+tv7OzhmyQN88EdLAbjiF0+za+87vs9n93qQ7qaQzsRddFxIRVEiINXivqb9LQAuunlpr/TpX7ybDTv2dP9+e38n//CTZb7PZyff3Z67wxn7NPauhE3z0P4M7d+Hq08/OG5TlBhJdcx9SP+c+cbkZlKa3DS4e6z1qMiHezQsoySF/g31LP/SKXGbocRMqsW9s0DIN+3cy9v7O5k6enCkNhw0Zgj1dcIVJ06tvjEallEUJRoyI+7/dvuLrNryFt9437tCO59d79Wh/RtY87XTHewbhkWKoij2pDrmXijuq7bk4u9X/f75uMxRFEVJDKkWd41zK4qi2JNqce902EIlSegAU4qiREHKxT1az334QGfzeSqKosRNqsU9yrDMs188ieEOJ2u241JrsoqDxwwJyiRFUZSypFrcw/bcrzq1pxNIo0+v/aTpzaxbsoCRg3XSZ0VRwifd4h6y5/6Bd08I9fiK4gWng+AptU2627l3hifu65YsAODc2eM56ZDm0M6jKG556J/msXnX3rjNUBJOqsW9w+mALi7o16f3x8y3zj888HMoyeGwCY1xm+Ca5qH9IxuvXkkvqRb33fs6HW33p08ey1nfewRj4Lhpo1h0VCsiMGnUIM774eNs252bW/WGi47g5OnNtkP7Kunl0mNaGTusP1+76yUA/nXBIXz1zpUAjG/UcdiVbJJqcX97f0fVbWa2DONdLcN45esLbNcfMHIg23bv59Pzp3HKoWOCNlFxyfUXzmLk4OCanA4f2MC/nXkoAA+93M6qLW9x3hET+OqdK1lw2NjAzqOkn2+dP5OJIwbGbUZgpFrcd+/v7bkff2ATf/lbO58//RD+/a6cZ9Y0pPLn6xfOmM41d6zg4/OmhGan4pwzZ44L7Fi/vnwuraMGdf/+xUfndi/n61QUJc+5s1viNiFQxG4wrKhpa2szy5a5H299w449rG1/izFD+3PPis1ccULPyIydXYZv3v0yHzluEqO0+aGiKBlERJ4yxrTZrQvFcxeRU4FvA/XAj40xS8I4z/jGAd0x02nNvTsH1dcJi0/TyQoURalNAm/nLiL1wPeA04DpwIUiMj3o8yiKoijlCaMT0xxgtTFmrTFmP/ArYGEI51EURVHKEIa4jwfWF/x+3UrrhYhcLiLLRGRZe3t7CGYoiqLULrENP2CMudEY02aMaWtqaorLDEVRlEwShrhvAAoHZWmx0hRFUZSICEPc/wpME5FJItIXuAC4PYTzKIqiKGUIvCmkMaZDRD4J3E2uKeTNxpgXgz6PoiiKUp5Q2rkbY+4C7grj2IqiKEp1EtFDVUTagVc97j4K2BqgOUGhdrkjqXZBcm1Tu9yRRbsOMMbYtkhJhLj7QUSWlet+GydqlzuSahck1za1yx21ZleqZ2JSFEVR7FFxVxRFySBZEPcb4zagDGqXO5JqFyTXNrXLHTVlV+pj7oqiKEopWfDcFUVRlCJU3BVFUTJIqsVdRE4VkZdFZLWILI743BNE5EERWSEiL4rIp630L4nIBhF51vo7vWCff7FsfVlETgnRtnUi8rx1/mVW2ggRuVdEVln/h1vpIiLfsexaLiKzQ7LpoII8eVZEdonIZ+LILxG5WUS2iMgLBWmu80dEFlnbrxKRRSHZ9U0Reck69x9FpNFKbxWRPQX59sOCfY6w7v9qy3ZfM76Xscv1fQv6eS1j168LbFonIs9a6VHmVzltiLaMGWNS+UduaIM1wGSgL/AcMD3C848FZlvLQ4C/kZuc5EvA52y2n27Z2A+YZNleH5Jt64BRRWn/ASy2lhcD37CWTwf+DAgwF1ga0b3bBBwQR34BxwOzgRe85g8wAlhr/R9uLQ8Pwa6TgT7W8jcK7Got3K7oOE9atopl+2kh2OXqvoXxvNrZVbT+WuCLMeRXOW2ItIyl2XOPdVIQY8xGY8zT1vLfgZXYjFtfwELgV8aYfcaYV4DV5K4hKhYCt1jLtwBnF6T/1OR4AmgUkbEh2zIfWGOMqdQrObT8Msb8BXjT5nxu8ucU4F5jzJvGmO3AvcCpQdtljLnHGNNh/XyC3CirZbFsG2qMecLkFOKnBdcSmF0VKHffAn9eK9lled/nA7+sdIyQ8qucNkRaxtIs7o4mBYkCEWkFZgFLraRPWp9XN+c/vYjWXgPcIyJPicjlVlqzMWajtbwJaI7BrjwX0Puhizu/wH3+xJFv/0DOw8szSUSeEZGHReQ4K228ZUsUdrm5b1Hn13HAZmPMqoK0yPOrSBsiLWNpFvdEICKDgd8DnzHG7AJ+AEwBDgc2kvs0jJpjjTGzyc1je4WIHF+40vJQYmkDK7lhoM8CfmslJSG/ehFn/pRDRD4PdAC3WkkbgYnGmFnAlcAvRGRohCYl7r4VcSG9HYjI88tGG7qJooylWdxjnxRERBrI3bxbjTF/ADDGbDbGdBpjuoAf0RNKiMxeY8wG6/8W4I+WDZvz4Rbr/5ao7bI4DXjaGLPZsjH2/LJwmz+R2ScilwBnAB+yRAEr7LHNWn6KXDz7QMuGwtBNKHZ5uG9R5lcf4Fzg1wX2RppfdtpAxGUszeIe66QgVkzvJmClMeZbBemF8epzgHxN/u3ABSLST0QmAdPIVeQEbdcgERmSXyZXIfeCdf58bfsi4LYCuy62auznAjsLPh3DoJdHFXd+FeA2f+4GThaR4VZI4mQrLVBE5FTgn4GzjDFvF6Q3iUi9tTyZXP6stWzbJSJzrTJ6ccG1BGmX2/sW5fP6XuAlY0x3uCXK/CqnDURdxvzUCsf9R66W+W/k3sKfj/jcx5L7rFoOPGv9nQ78DHjeSr8dGFuwz+ctW1/GZ418Bbsmk2uJ8BzwYj5fgJHA/cAq4D5ghJUuwPcsu54H2kLMs0HANmBYQVrk+UXu5bIReIdcHPMyL/lDLga+2vq7NCS7VpOLu+bL2A+tbd9n3d9ngaeBMwuO00ZObNcA38XqiR6wXa7vW9DPq51dVvpPgI8VbRtlfpXThkjLmA4/oCiKkkHSHJZRFEVRyqDiriiKkkFU3BVFUTKIiruiKEoGUXFXFEXJICruiqIoGUTFXVEUJYP8Lw9J2tnimonfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用Matplotlib绘制奖励值的曲线图\n",
    "plt.plot(episode_rewards)\n",
    "plt.title(\"reward\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f5ab5837",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM8AAACeCAYAAACVU14NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMv0lEQVR4nO3de4xc5X3G8e8ze7HBDjZObJdiCBebWG6bErOloPQPRBLVcS+2VBpBUEJTS7QqkUhF2tJWakLUlkRKIY2KUrkljVOhEkpuQGlRuDRVK3GxSeoEI/DimtZgLjbG2F7D7uz8+sd514yXdXb29R7OzPHzkUZ7znvOzP7eWT+eM++ceY8iAjObuUbVBZj1KofHLJPDY5bJ4THL5PCYZXJ4zDI5PF1C0kFJ58zWvpIukbRrdqoDSSFp+Ww9Xh30V13AiUjSTmApMN7WfF5EPN/J/SNifhl12cw4PNX5tYi4v+oiLJ8P27pE+2GRpK9JukXSv0g6IOkRSeceY9+1kral/Z6T9OlJj3udpJck7Zb0ibb2OZK+KOl/Jb0o6W8lndS2/Q/SfZ6X9NvlPwO9x+HpXpcDNwCnAsPAXxxjv1uB34mIdwA/CzzYtu2ngAXA6cAG4BZJp6ZtnwfOA84Hlqd9/gxA0hrg08CHgBXAB2erU3Xi8FTnO5JeTbfvTLH92xHxaEQ0gdso/pFPZQxYJemUiNgXEY9P2va5iBiLiHuBg8B7JAm4Gvj9iHglIg4Af0kRWICPAP8QET+OiEPAZ4+zr7Xk8FRnfUQsTLf1U2x/oW15BDjWIMFvAGuBZyV9X9LFbdv2pvBNfpzFwMnAlokAA/+W2gF+Gvi/tvs922GfTigeMOhxEfEYsE7SAPBJ4A7gjGnutgc4DPxMRDw3xfbdkx7jzNmotW78ytPDJA1KulLSgogYA14DWtPdLyJawN8BN0takh7rdEm/nHa5A/gtSasknQx8pqQu9DSHp/d9DNgp6TXgd4ErO7zfH1EMRDyc7ns/8B6AiPhX4EsUgw/DHD0IYYn8ZTizPH7lMctUSngkrZH0lKRhSdeX8TvMqjbrh22S+oCnKT5g2wU8BlwREdtm9ReZVayMV54LgeGI2BERo8DtwLoSfo9Zpcr4nOd0jv6AbRfwi5N3knQ1xafczJs374KVK1eWUIrZ8dm5cyd79uzRVNsq+5A0IjYCGwGGhoZi8+bNVZVidkxDQ0PH3FbGYdtzHP3p9LLUZlYrZYTnMWCFpLMlDVKcbHhXCb/HrFKzftgWEU1JnwTuA/qAr0bEE7P9e8yqVsp7nnT6+71lPLZZt/AZBmaZHB6zTA6PWSaHxyyTw2OWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8NjlsnhMcvk8JhlcnjMMjk8ZpkcHrNMDo9ZpmnDI+mr6YKwP25rWyTpe5K2p5+npnZJ+nKao3qrpNVlFm9WpU5eeb4GrJnUdj3wQESsAB5I6wAfprgA7AqK2UC/MjtlmnWfacMTEf8BvDKpeR2wKS1vAta3tX89Cg8DCyWdNku1mnWV3Pc8SyNid1p+AVialqeap/r0qR5A0tWSNkva/PLLL2eWYVad4x4wiOIaJTO+TklEbIyIoYgYWrx48fR3MOsyueF5ceJwLP18KbV7nmo7YeSG5y7gqrR8FfDdtvaPp1G3i4D9bYd3ZrUy7XS7kv4JuAR4l6RdFJcV/zxwh6QNwLPAR9Lu9wJrKa6gPAJ8ooSazbrCtOGJiCuOsekDU+wbwDXHW5RZL/AZBmaZHB6zTA6PWSaHxyyTw2OWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8Njlmnas6qtt0QERKv4CajRQPL/kWVweGro2f+6nYMvPgPAsl9Yz8J3v7fiiurJ4amZaI1zeN/zHN67C4DmGyMVV1Rffj2vm2jRao4dWZUqrKXmHJ6aaTVHGRvZD4AafQycvLDagmrM4amZiCDGm8WKGvTNObnagmqsk7mqz5D0kKRtkp6QdG1q93zVXU4Sjf7BqsuorU5eeZrAdRGxCrgIuEbSKjxfdVeKaHFkDkqJRqOv0nrqrJO5qndHxONp+QDwJMUUup6vugtFc4xotdpaPGJQlhm955F0FvA+4BGOc75qz1VdjlZzlIjxtObglKnj8EiaD3wT+FREvNa+LWe+as9VXY7Rkf20mqMA9M+dT9/g3Iorqq+OwiNpgCI4t0XEt1Kz56vuRq1xSKfm9PUPor6Biguqr05G2wTcCjwZETe1bfJ81V1Off3IAwal6eT0nPcDHwN+JOmHqe1P8HzVXanVah5ZlhrIpxiUppO5qv+TY7/z9HzVXaY19kbVJZwwfIZBzYyPHn5zpeE/b5n87NbMG6+9Oew/95TFPjO0RA5PzRRnGBT6Bk/Cn/WUx+GpkYlvj05o9M+pqJITg8NTM63xttE2v+cplZ/dWom3jLZ5qLo8Dk+dRDB2+M0zp/oGTqqwmPpzeGokJoVncP6iCqupP4entkRjwAMGZXJ4auXo0ba+AX+LtEwOT41Eq0WMj7/Z4MkOS+Vnt0ai1STaTgy1cjk8NTI++vqRc9vU10/fgL8IVyaHp0ZazVHG0+c8jf5B+ufMq7iienN4akqNBur3t0jL5PDUypujbVKDRsNTkZfJ4amRYo7qtuFqn5lTKv/X1IMOHTrExo0bGRk5+goIiwZHWf2OcRqCAwcOctNf3cRYHP3/4+WXX8655577dpZbWw5PDzp06BA33ngjk+e7++DQeSz66O8xGvNp7v13PvPFz/H66NFD16tXr3Z4Zkkns+fMlfSopP9Oc1XfkNrPlvRImpP6G5IGU/uctD6ctp9Vch8sOaDlPD1yMf9z+L1sfvnnaI63pr+TZevkPc8bwKUR8fPA+cCaNKXUF4CbI2I5sA/YkPbfAOxL7Ten/ext0K9RGowDQat5kPHWjOahtBnqZPacAA6m1YF0C+BS4KOpfRPwWYpJ3delZYA7gb+RpJj8Ncc2IyMjbNmyJaP8E9O+fftoNt96JsHuXY/z/ftuYDROYmz/1rd8sxRg+/btLFmy5O0osxYmv69s19F7Hkl9wBZgOXAL8AzwakRM/AXb56M+Mld1RDQl7QfeCeyZ9JhXU1xFgWXLlnHmmWd22B2bN28ejSm+Jbrj+VfY8fx9P/G+S5Ys8XM9A4ODxz65tqPwRDFz+PmSFgLfBlYeb1ERsRHYCDA0NBSer7pzETFleDqxYMEC/Fx3rr//2BGZ0V8gIl4FHgIuprh0yMQjt89HfWSu6rR9AbB3RhWb9YBORtsWp1ccJJ0EfIjiGj0PAZel3SbPVT0xh/VlwIM/6f2OzZwk+vv7Z3wbGBjwnAazqJPDttOATel9TwO4IyLukbQNuF3SnwM/oJgMnvTzHyUNA68Al5dQ9wlt4cKF3H333YyNjU2/8yQrVx73EbclnYy2baW4oNXk9h3AhVO0vw785qxUZ1MaGBjgggsuqLqME57PbTPL5PCYZXJ4zDI5PGaZHB6zTA6PWSaHxyyTw2OWyeExy+TwmGVyeMwyOTxmmRwes0wOj1kmh8csk8NjlsnhMcvk8JhlcnjMMjk8ZpkcHrNMDo9ZJnXDfISSDgBPVV1Hid7FpLm6a6bO/Xt3REw5P3G3XNzqqYgYqrqIskja7P7Vjw/bzDI5PGaZuiU8G6suoGTuXw11xYCBWS/qllces57j8Jhlqjw8ktZIeipdev76quuZKUlnSHpI0jZJT0i6NrUvkvQ9SdvTz1NTuyR9OfV3q6TV1fagM5L6JP1A0j1p/WxJj6R+fEPSYGqfk9aH0/azKi28RJWGJ10w6xbgw8Aq4ApJq6qsKUMTuC4iVgEXAdekPlwPPBARK4AH0joUfV2RbldTXEG8F1xLcUXACV8Abo6I5cA+YENq3wDsS+03p/1qqepXnguB4YjYERGjwO0Ul6LvGRGxOyIeT8sHKP6BnU7Rj01pt03A+rS8Dvh6FB6muLbraW9v1TMjaRnwK8Dfp3UBlwJ3pl0m92+i33cCH1BNr+VYdXiOXHY+ab8kfc9JhyjvAx4BlkbE7rTpBWBpWu7FPn8J+EOgldbfCbwaEc203t6HI/1L2/en/Wun6vDUhqT5wDeBT0XEa+3b0gWNe/IzAUm/CrwUEVuqrqXbVH1u25HLziftl6TvGZIGKIJzW0R8KzW/KOm0iNidDsteSu291uf3A78uaS0wFzgF+GuKw83+9OrS3oeJ/u2S1A8sAPa+/WWXr+pXnseAFWnkZpDiytl3VVzTjKTj+VuBJyPiprZNdwFXpeWrgO+2tX88jbpdBOxvO7zrOhHxxxGxLCLOovj7PBgRVwIPAZel3Sb3b6Lfl6X9e/JVd1oRUekNWAs8DTwD/GnV9WTU/0sUh2RbgR+m21qK4/wHgO3A/cCitL8oRhifAX4EDFXdhxn09RLgnrR8DvAoMAz8MzAntc9N68Np+zlV113WzafnmGWq+rDNrGc5PGaZHB6zTA6PWSaHxyyTw2OWyeExy/T/Nx/5HWljVrkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 重置环境，开始新的一轮游戏\n",
    "observation, _ = env.reset()\n",
    "# 创建GymHelper对象来辅助显示\n",
    "gym_helper = GymHelper(env, figsize = (3, 3))\n",
    "\n",
    "# 开始游戏\n",
    "for i in range(700):\n",
    "    # 渲染环境，title为当前步骤数\n",
    "    gym_helper.render(title = str(i))\n",
    "    \n",
    "    # 通过Q网络找到当前状态下的最优动作\n",
    "    action = agent.choose_action(observation)\n",
    "    # 执行action，获取新的信息\n",
    "    observation, reward, terminated, truncated, info = env.step(action)\n",
    "    \n",
    "    # 如果游戏结束，则结束当前循环\n",
    "    if terminated or truncated:\n",
    "        break\n",
    "\n",
    "# 游戏结束\n",
    "gym_helper.render(title = \"Finished\")\n",
    "# 关闭环境\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16644ec8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
